package com.chance.cc.crawler.development.scripts.pcauto.carSeriesDistributor;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class PcautoCarSeriesDistributorCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(PcautoCarSeriesDistributorCrawlerScript.class);//日志
    private static final String DOMAIN = "pcauto";
    private static final String SITE = "carSeriesDistributor";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //https://www.pcauto.com.cn/global/1603/intf8771.js
    private static final String City = "https://www.pcauto.com.cn/global/1603/intf8771.js";//动态获取城市链接
    //https://price.pcauto.com.cn/api/hcs/select/compareNewBar/serial_brand_json_chooser?type=1
    private static final String FirstUrl = "https://price.pcauto.com.cn/api/hcs/select/compareNewBar/serial_brand_json_chooser\\?type=1";//动态获取品牌网址

    //https://price.pcauto.com.cn/shangjia/c3/nb4/p2.html
    private static final String CarSeriesRegulars = "https://price.pcauto.com.cn/shangjia/\\S*/\\S*/p\\d*.html";//车系链接
    //https://price.pcauto.com.cn/98991/
    private static final String ShopRegulars = "https://price.pcauto.com.cn/\\d*/";//门店链接
    //https://price.pcauto.com.cn/98991/p1/market.html
    private static final String PromotionRegulars = "https://price.pcauto.com.cn/\\d*/p\\d*/market.html\\S*";//促销链接
    //https://price.pcauto.com.cn/market/98991-103212210.html#ad=8796
    //https://price.pcauto.com.cn/market/1069869-103704550.html#ad=8796
    private static final String ContentRegulars = "https://price.pcauto.com.cn/market/\\S*.html\\S*";//信息链接

    @Override
    public String domain() {
        return DOMAIN;
    }//脚本domain定义:识别具体的脚本

    @Override
    public void initUrlRegulars() {//进入脚本的正则列表
        addUrlRegular(City);
        addUrlRegular(FirstUrl);
        addUrlRegular(CarSeriesRegulars); //下一页
        addUrlRegular(ShopRegulars); //门店
        addUrlRegular(PromotionRegulars);//促销
        addUrlRegular(ContentRegulars);//新闻资讯
    }

    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }//是否执行脚本 ： 输入数据检查，合格的才进入脚本

    //获取客户需要的信息
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();//获取页面
        Json json = internalDownloadPage.getJson();//获取json
        String msg = json.jsonPath($_type + ".msg").get();//判断是否成功
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        try {
            Map<String, List<String>> cityAndBrandIdMap = new HashMap();//保存用户信息
            List<String> brandIdList = new ArrayList<>();//保存品牌
            List<String> all1 = json.jsonPath($_type + ".content").all();//获取内容
            for (String data : all1) {//遍历key
                JSONObject jsonObject = JSONObject.parseObject(data);
                String keyword = jsonObject.getString("keyword");//获取key值
                String[] trimUrl = Pattern.compile("[^0-9a-zA-z\\u4e00-\\u9fa5]").matcher(keyword).replaceAll(" ").replaceAll("\\s+", " ").replaceAll("\\[", "").replaceAll("\\]", "").trim().split(" ");//获取品牌id
                brandIdList.clear();//清空汽车
                String city = trimUrl[0];
                for (int i = 1; i < trimUrl.length; i++) {//保存城市和汽车
                    if (trimUrl[i] != "") {
                        brandIdList.add(trimUrl[i]);
                    }
                }
                cityAndBrandIdMap.put(city, brandIdList);
            }
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("cityAndBrandId", cityAndBrandIdMap);//用户需求放入record中
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    //解析链接方法
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();//创建脚本数组
//        判断页面是否加载成功
        String requestUrl = crawlerRequestRecord.getHttpRequest().getUrl();
        if (httpPage.getStatusCode() != 200 || (!httpPage.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, httpPage.getStatusCode());
            if (httpPage.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRequestRecord);
                crawlerRequestRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }

        Map<String, String> carNames = new HashMap();//保存动态获取汽车
        Map<String, String> cityNames = new HashMap();//保存动态获取城市
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(FirstUrl) || crawlerRequestRecord.getHttpRequest().getUrl().matches(City)) {//解析动态网页
            request(crawlerRequestRecord, httpPage, parsedLinks, carNames, cityNames);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {//解析门店
            shopRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(ShopRegulars)) {//解析促销
            promotionRequest(crawlerRequestRecord, httpPage, parsedLinks);
        } else if (crawlerRequestRecord.getHttpRequest().getUrl().matches(PromotionRegulars)) {//解析内容
            contentRequest(crawlerRequestRecord, httpPage, parsedLinks);
        }
        return parsedLinks;
    }

    //解析动态地址和品牌
    public void request(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, Map<String, String> carNames, Map<String, String> cityNames) {//促销方法
        //        解析汽车品牌名
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(FirstUrl)) {
            Json json = new Json(httpPage.getRawText().substring(21, httpPage.getRawText().length() - 1).trim());//获取json文本
            List<String> carName1 = json.jsonPath($_type + "..[*].NAME").all();//品牌名
            List<String> carBrand1 = json.jsonPath($_type + "..[*].ID").all();//品牌id
            for (int i = 0; i < carName1.size(); i++) {//保存到map数组中
                carNames.put(carName1.get(i), "nb" + carBrand1.get(i));
            }
            List<String> carName2 = json.jsonPath($_type + "..[*].LIST..NAME").all();//品牌名
            List<String> carBrand2 = json.jsonPath($_type + "..[*].LIST..ID").all();//品牌id
            for (int i = 0; i < carName2.size(); i++) {//保存到map数组中
                carNames.put(carName2.get(i), "b" + carBrand2.get(i));
            }
            List<String> carName3 = json.jsonPath($_type + "..[*].LIST..LIST..NAME").all();//品牌名
            List<String> carBrand3 = json.jsonPath($_type + "..[*].LIST..LIST..ID").all();//品牌id
            for (int i = 0; i < carName3.size(); i++) {//保存到map数组中
                carNames.put(carName3.get(i), "sg" + carBrand3.get(i));
            }
            crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("carNames", carNames);//保存品牌信息
            String cityUrl = "https://www.pcauto.com.cn/global/1603/intf8771.js";//动态城市链接
            CrawlerRequestRecord cityRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .recordKey(cityUrl)
                    .httpUrl(cityUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()//拷贝上一个自定义标签
                    .copyResultTags()
                    .build();
            cityRecord.getHttpRequest().setResponseCharset("GBK");
            parsedLinks.add(cityRecord);
        }

        //        解析城市名
        if (crawlerRequestRecord.getHttpRequest().getUrl().matches(City)) {
            KVTag carnames = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("carNames");//获取动态汽车名
            Map<String, String> carNamesVal = (Map<String, String>) carnames.getVal();

            Json json = new Json(httpPage.getRawText().substring(16, httpPage.getRawText().length() - 1033).trim());//获取json文本
            List<String> name1 = json.jsonPath($_type + "..[*].name").all();//城市名
            List<String> pinyin1 = json.jsonPath($_type + "..[*].id").all();//城市id
            for (int i = 0; i < name1.size(); i++) {//保存到map数组中
                cityNames.put(name1.get(i), "p" + pinyin1.get(i));//省
            }
            List<String> name2 = json.jsonPath($_type + "..[*].city..name").all();//城市名
            List<String> pinyin2 = json.jsonPath($_type + "..[*].city..id").all();//城市id
            for (int i = 0; i < name2.size(); i++) {//保存到map数组中
                cityNames.put(name2.get(i), "c" + pinyin2.get(i));//市+行政区
            }
            List<String> name3 = json.jsonPath($_type + "..[*].city..district..name").all();//城市名
            List<String> pinyin3 = json.jsonPath($_type + "..[*].city..district..id").all();//城市id
            for (int i = 0; i < name3.size(); i++) {//保存到map数组中
                cityNames.put(name3.get(i), "a" + pinyin3.get(i));//县或区
            }

            KVTag cityAndBrandId = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("cityAndBrandId");//获取用户信息
            Map<String, List<String>> brandIds = (Map<String, List<String>>) cityAndBrandId.getVal();


            for (String city : brandIds.keySet()) {//key用户城市名
                String cityId = cityNames.get(city);//城市名转换拼音
                List<String> brands = brandIds.get(city);//品牌
                for (int i = 0; i < brands.size(); i++) {
                    String brandNumber = carNamesVal.get(brands.get(i));//品牌转换为id数字
                    if (cityId == null || brandNumber == null) {
                        log.error("没有这个城市或品牌 cityId:" + cityId + "city:" + city + " brandNumber:" + brandNumber);
                    } else {
                        //https://price.pcauto.com.cn/shangjia/c3/nb571/
                        String url = "https://price.pcauto.com.cn/shangjia/" + cityId + "/" + brandNumber + "/p1.html";//得到进入汽车之家主页面
                        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .httpUrl(url)
                                .releaseTime(Long.valueOf(System.currentTimeMillis()))
                                .copyBizTags()
                                .copyResultTags()
                                .build();
                        parsedLinks.add(record);
                    }
                }
            }
        }
    }

    //    主页面解析
    public static void shopRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {//促销方法
        //          下一页解析
        String u = httpPage.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get();
        if (u != null) {
            String nextPageUrl = "https://price.pcauto.com.cn" + u;//网址
            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(turnPageRequest);
        }

        //          门店解析
        String brandUrl = httpPage.getRequest().getUrl();//获取url
        String[] trim = brandUrl.split("/");//截取网址
        String brandId = trim[trim.length - 2];//品牌id
//        String cityId = trim[trim.length-3];//城市id
        List<String> var = httpPage.getHtml().xpath("//a[@class=\"tag s\"]").all();//城市
        String city = var.get(0);//城
        String brand = var.get(1);//品牌
        Map tagBrandDetail = new HashMap();
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_name, brand);//品牌名
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_id, brandId);//品牌id
        tagBrandDetail.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url

        List<Selectable> nodes = httpPage.getHtml().xpath("//li[@class=\"liFc\"]").nodes();//所有门店
        if (nodes.size() > 0) {
            for (Selectable node : nodes) {//获取单个门店
                String state = node.xpath(".//span[@class=\"icon icon-jxs-green\"]").get();//状态
                if (state != null) {
                    String su = node.xpath(".//div[@class=\"divYSd\"]/p/a/@href").get();//店铺url
                    if (su != null || su != "") {
                        String shopId = su.replaceAll("/", "");//门店id
                        String shopUrl = "https://price.pcauto.com.cn" + su;//店铺url
                        String shopName = node.xpath(".//div[@class=\"divYSd\"]/p/a/strong/text()").get();//店铺名
                        String shopPhone = node.xpath(".//strong[@class=\"red\"]/text()").get();//电话
                        String shopAddress = node.xpath(".//p[@class=\"pFc\"]/span/@title").get();//地址
                        Map tagShopDetail = new HashMap();
                        tagShopDetail.put(AutoVMCommonField.Field_Shop_id, shopId);
                        tagShopDetail.put(AutoVMCommonField.Field_Shop_Url, shopUrl);
                        tagShopDetail.put(AutoVMCommonField.Field_Shop_Name, shopName);
                        tagShopDetail.put(AutoVMCommonField.Field_Shop_Phone, shopPhone);
                        tagShopDetail.put(AutoVMCommonField.Field_Shop_Address, shopAddress);

                        CrawlerRequestRecord shopRecord = CrawlerRequestRecord.builder()
                                .itemPageRequest(crawlerRequestRecord)
                                .recordKey(shopUrl)
                                .httpUrl(shopUrl)
                                .releaseTime(System.currentTimeMillis())
                                .copyBizTags()
                                .copyResultTags()
                                .build();
//                shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_id, seriesId);
                        shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_City, city);
                        shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Brand_detail, tagBrandDetail);
                        shopRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Shop_detail, tagShopDetail);
                        shopRecord.tagsCreator().bizTags().addCustomKV("shopUrl", shopUrl);
                        parsedLinks.add(shopRecord);
                    }
                }
            }
        }
    }

    //  促销解析
    public static void promotionRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String promotionUrl = httpPage.getRequest().getUrl() + "p1/market.html";//促销链接
        CrawlerRequestRecord promotionRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .recordKey(promotionUrl)
                .httpUrl(promotionUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .build();
        promotionRecord.tagsCreator().bizTags().addCustomKV("promotionUrl", promotionUrl);
        parsedLinks.add(promotionRecord);
    }


    //  详情优惠解析
    public static void contentRequest(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        String nextPreferentialPageUrl = httpPage.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get(); //优惠下一页
        if (nextPreferentialPageUrl != null && nextPreferentialPageUrl!="") {
            CrawlerRequestRecord nextPreferentialPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .httpUrl(nextPreferentialPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .recordKey(nextPreferentialPageUrl)
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(nextPreferentialPageRequest);
        }

        //  优惠解析
        List<Selectable> nodes = httpPage.getHtml().xpath("//ul[@class=\"cxyhlt clearfix\"]//li").nodes();//所有优惠
        for (Selectable node : nodes) {//获取单个优惠url
            String detailsPreferentialUrl = node.xpath("./a/@href").get();//优惠页面链接
            if (StringUtils.isBlank(detailsPreferentialUrl)) {
                continue;
            }
            String overTime = node.xpath(".//font[@class=\"red\"]/text()").get();//获取优惠结束时间
            String pubTime = node.xpath(".//div[@class=\"tagcont\"]/span/text()").get().substring(6);//获取时间
            if (StringUtils.isBlank(pubTime)) {
                continue;
            }
            //保存url
            String shopUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("shopUrl");
            String promotionUrl = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("promotionUrl");
            ArrayList<Map<String, String>> tagCategroyList = new ArrayList<>();
            HashMap<String, String> tagCategroyMap = new HashMap<>();
            tagCategroyMap.put("shopUrl", shopUrl);
            tagCategroyMap.put("promotionUrl", promotionUrl);
            tagCategroyMap.put("detailsPreferentialUrl", detailsPreferentialUrl);
            tagCategroyList.add(tagCategroyMap);

            try {
                CrawlerRequestRecord contentPreferentialRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .recordKey(detailsPreferentialUrl)
                        .httpUrl(detailsPreferentialUrl)
                        .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime())
                        .needWashed(true)
                        .copyBizTags()//拷贝上一个自定义标签
                        .copyResultTags()
                        .resultLabelTag(article)
                        .build();
                contentPreferentialRecord.tagsCreator().bizTags().addCustomKV("pubTime", pubTime);//结束时间
                contentPreferentialRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Remaining_Time, overTime);//结束时间
                contentPreferentialRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Categroy_List, tagCategroyList);//全部标签
                parsedLinks.add(contentPreferentialRecord);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
    }


    //  过滤冲洗页面
    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();//解析

        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(washCustomNewCar(crawlerRecord, page));
            }
        }
        return crawlerDataList;
    }


    //内容清洗
    public List<CrawlerData> washCustomNewCar(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        log.info("内容清洗");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        //新闻资讯
        try {
            String detailsInformationUrl = httpPage.getRequest().getUrl();
            String activity = httpPage.getHtml().xpath("//div[@class=\"art-gray-info\"]/p/span/text()").get();//报价
            if (activity == null) {
                activity = httpPage.getHtml().xpath("//div[@class=\"redtit\"]/text()").get();//或促销时间
            }
            String condition = httpPage.getHtml().xpath("//span[@class=\"mosmtext\"]/text()").get();//获取文本
            String picture = httpPage.getHtml().xpath("//div[@class=\"art-info-left\"]//img/@src").get();//获取图片
            if (picture == null) {
                picture = httpPage.getHtml().xpath("//div[@class=\"pictsr\"]//img/@src").get();//获取图片
            }
            String title = httpPage.getHtml().xpath("//p[@class=\"titsmp\"]/text()").get();//标题
            String pubTime = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("pubTime");//时间
            String[] split = detailsInformationUrl.replaceAll(".html", "/").split("/");

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", domain(), article, split[split.length - 2]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(detailsInformationUrl)
                    .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd").getTime())
                    .addContentKV(AICCommonField.Field_Title, title)
                    .addContentKV(AICCommonField.Field_Content, condition)
                    .addContentKV(AICCommonField.Field_Images, picture)
                    .addContentKV("activity", activity)
                    .build();
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    //最后执行的方法
    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 50) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

}